/* 

==========================================================

DX490a - Summer 2010

Instructor: Stelios Manousakis

==========================================================

Class 8.1:

Interfacing 4: Feature extraction in SuperCollider

Contents:

• About

• Feature extraction in SC

- Amplitude

- Pitch

- Time-related

- timbre

==========================================================

*/

// ================= ABOUT =================

/* 

• Psychological dimensions of sound:

- loudness

- pitch

- duration

- timbre

• Data you can get:

- continuous: description of a particular characteristic in time.

- switches/triggers: detection of specific behaviors, energy in specific areas, etc

• Analysis domains:

- analysis of the psychological dimensions of sound (pshycoacoustic analysis / feature extraction)

- analysis of organizational qualities through gestalt-driven analysis (cognitive analysis / machine listening)

> Our brains attempt to make sense of our noisy world, compressing the torrent of incoming sensual data by finding symmetry, order, simplicity and patterns. A set of laws has been postulated by Gestalt psychologists to explain different strategies our brain uses to group objects together:

proximity, closure, similarity, continuity, symmetry, common fate

"open http://www.infovis-wiki.net/index.php?title=Gestalt_Laws".unixCmd

*/

// ================= FEATURE EXTRACTION IN SUPERCOLLIDER =================

// There are really a lot of UGens that analyze signals in SC. You would most commonly use those for audio, but they can be quite effective for any kind of incoming signal.

/* These examples are very basic, but you can refer to:

- class example 2.1_[Server-language communication] for receiving analysis data in sclang, and 

- class example 6.2_[Mapping] about mapping strategies

*/

// ====== AMPLITUDE ======

// ------ Examples  --

// The most fundamental UGen is Amplitude:

// • continuous:

(

a = CtkSynthDef(\ampTest, {arg inbus, attack, decay;

var amp, src;

src = WhiteNoise.ar(0.5);

amp = Amplitude.ar(In.ar(inbus), attack, decay);

amp.poll;

Out.ar(0, src * amp);

});

)

// try out a few different configuration of the analysis UGen

b = a.new.inbus_(s.options.numInputBusChannels).attack_(0.01).decay_(0.01).play;

b.free;

b = a.new.inbus_(s.options.numInputBusChannels).attack_(0.1).decay_(1).play;

b.free;

b = a.new.inbus_(s.options.numInputBusChannels).attack_(1.0).decay_(1).play;

b.free;

// • triggers:

// You can get amplitude data into sclang, and do something with it.

/* But:

- use Trig1 to limit the nr of data you get!

- allow yourself to set the trigger threshold and Trig1 'silent' duration from outside

*/

// This tracks incoming audio from your soundcard - beware of feedback 

(

a = CtkSynthDef(\ampTest, {arg inbus, attack, decay, id, thresh, tdur;

var amp;

amp = Amplitude.ar(In.ar(inbus), attack, decay);

SendTrig.ar(Trig1.ar(amp > thresh, tdur), id, amp); 

});

c = CtkSynthDef(\sin, {arg outbus = 0, freq = 220;

Out.ar(0, SinOsc.ar(440, 0, XLine.kr(0.1, 0.0001, 1, doneAction: 2)))

});

o = OSCresponderNode(s.addr, '/tr', {arg time, resp, msg;

[time, resp, msg].postln;

c.new.play

}).add; 

)

b = a.new.inbus_(s.options.numInputBusChannels).attack_(0.01).decay_(0.01).id_(100).thresh_(0.15).tdur_(1).play;

o.remove;

b.free;

// ------ A list of AMPLITUDE tracking UGens in SC --

// >> Amplitude following

Amplitude // amplitude follower

PeakFollower // peak amp tracker

EnvDetect // amplitude follower

WAmp

// >> Amplitude minimum/maximum 

Peak

DetectSilence

// >> Amplitude utilities

// pshychoacoustic utilities

AmpComp / AmpCompA

Loudness // get loudness in sones

// >> Signal operations:

Compander

Limiter

Normalizer // normalize to nominal level

// ====== PITCH ======

// ------ Examples  --

// These examples use the incoming audio from your soundcard - beware of feedback 

// • continuous:

// Using the Pitch UGen: pretty good!

(

a = CtkSynthDef(\pitch, {arg inbus, high = 1000, low = 60;

var in, freq, hasFreq;

in = In.ar(inbus, 1);

#freq, hasFreq = Pitch.kr(in, high, low, high);

freq.poll;

Out.ar(0, SinOsc.ar(freq * 4, 0, 0.1) * hasFreq)

});

)

b = a.new.inbus_(s.options.numOutputBusChannels).play;

b.free;

// The Tartini UGen is a more expensive, but is also better

(

a = CtkSynthDef(\pitch, {arg inbus;

var in, freq, hasFreq;

in = In.ar(inbus, 1);

#freq, hasFreq = Tartini.kr(in);

freq.poll;

Out.ar(0, SinOsc.ar(freq * 4, 0, 0.1))

});

)

b = a.new.inbus_(s.options.numOutputBusChannels).play;

b.free;

// • triggers:

// Triggering is much trickier with pitch, as precise pitch is close to impossible, so better to look for a specific region/register.

// This example will provide a trigger if the incoming audio is between 1500-1700Hz

(

a = CtkSynthDef(\pitch, {arg inbus, low = 1500, high = 1700, id, tdur = 0.5;

var in, freq, hasFreq, trig1, trig2, trig;

in = In.ar(inbus, 1);

#freq, hasFreq = Tartini.kr(in);

freq.poll;

trig1 = freq > low;

trig2 = freq < high;

trig = Trig1.kr((trig1 + trig2 - 1).poll, tdur);

SendTrig.kr(trig, id, freq);

});

b = a.new.inbus_(s.options.numOutputBusChannels).id_(100).play;

z = CtkSynthDef(\test, {arg freq;

Out.ar(0, SinOsc.ar(freq, 0, XLine.kr(0.1, 0.0001, 1, doneAction: 2)))

});

o = OSCresponderNode(s.addr, '/tr', {arg time, resp, msg;

[time, resp, msg].postln;

z.new.freq_(msg[3] * 0.5).play;

}).add; 

)

b.free;

o.remove; // don't forget to remove the responder

// ------ A list of PITCH tracking UGens in SC --

Pitch

Tartini

ZeroCrossing

// >> more/advanced/experimental (ie: use at your own risk):

KeyTrack

// ====== TIME-RELATED (ONSETS, ETC) ======

// ------ Examples  --

// There are a few different strategies for finding onsets. You can use some of the pre-cooked classes, or devise your own methods:

// These examples use the incoming audio from your soundcard - beware of feedback 

// comparing a signal with a delayed version of itself (an example by Josh Parmenter):

(

a = CtkSynthDef(\ampTest, {arg inbus, level = 0.5, attack, decay, id;

var amp, src, del, diff;

src = In.ar(inbus, 1);

amp = Amplitude.ar(src, attack, decay);

del = DelayN.ar(amp, 0.01, 0.01);

diff = (amp / del.max(0.001)).ampdb * 

EnvGen.kr(Env([0, 0, 1], [0.1, 0.1])).poll;

SendTrig.ar(Trig1.ar(diff > level, 1), id, diff);

});

z = CtkSynthDef(\test, {

Out.ar(0, SinOsc.ar(440, 0, XLine.kr(0.1, 0.0001, 1, doneAction: 2)))

});

o = OSCresponderNode(s.addr, '/tr', {arg time, resp, msg;

[time, resp, msg].postln;

// now - actually do something once the trigger happens

z.new.play;

}).add; 

)

b = a.new.inbus_(s.options.numOutputBusChannels).level_(36).attack_(0.01).decay_(0.01).id_(100).play;

b.level_(12); // level is in db

o.remove; 

b.free;

// or use the Slope UGen instead:

(

a = CtkSynthDef(\ampTest, {arg inbus, level = 0.5, attack, decay, id;

var amp, src, slope;

src = In.ar(inbus, 1);

amp = Amplitude.ar(src, attack, decay);

slope = Slope.ar(amp);

SendTrig.ar(Trig1.ar(slope > level, 1), id, slope);

});

z = CtkSynthDef(\test, {

Out.ar(0, SinOsc.ar(440, 0, XLine.kr(0.1, 0.0001, 1, doneAction: 2)))

});

o = OSCresponderNode(s.addr, '/tr', {arg time, resp, msg;

[time, resp, msg].postln;

// now - actually do something once the trigger happens

z.new.play;

}).add; 

)

b = a.new.inbus_(s.options.numOutputBusChannels).level_(0.2).attack_(0.01).decay_(0.01).id_(100).play;

b.level_(200); // angle?

o.remove; 

b.free;

// ------ A list of TIME (onset, rhythm, etc) tracking UGens in SC --

Onsets

Coyote

// >> more/advanced/experimental (ie: use at your own risk):

BeatTrack

BeatTrack2

AutoTrack

OnsetDS / XiiOnsets // no help file

SLOnset

Concat

// ====== TIMBRE ======

// There is a big variety of UGens that you can use for tracking characteristics on the spectrum of a signal in SC.

// ------ Examples --

// For example, SpecFlatness tells you how noisy a signal is (noise = 1, sine = 0)

(

f = Buffer.alloc(s,2048,1); // a buffer for the FFT to use

a = CtkSynthDef(\pitch, {arg inbus;

var in, freq, chain, flat;

in = In.ar(inbus, 1);

chain = FFT(f, in);

flat = SpecFlatness.kr(chain).poll;

flat = flat ** 3;

flat.poll;

Out.ar(0, BPF.ar(WhiteNoise.ar(0.5), 400, flat.max(0.0001)));

});

)

b = a.new.inbus_(s.options.numOutputBusChannels).play;

b.free;

// FFTPeak finds the bin with the strongest magnitude and outputs its frequency and magnitude:

(

f = Buffer.alloc(s, 2048, 1); // a buffer for the FFT to use

a = CtkSynthDef(\pitch, {arg inbus;

var in, chain, freq, mag;

in = In.ar(inbus, 1);

chain = FFT(f, in);

# freq, mag = FFTPeak.kr(chain).poll;

//peak = flat ** 3;

//flat.poll;

Out.ar(0, SinOsc.ar(freq, 0, mag * 0.001));

});

)

b = a.new.inbus_(s.options.numOutputBusChannels).play;

b.free;

// ------ A list of SPECTRAL FEATURE tracking UGens in SC --

SpecCentroid

SpecFlatness

SpecPcile

FFTCrest

FFTComplexDev

FFTFlux

FFTPeak

FFTRumble

FFTSlope

FFTSpread

FFTSubbandFlatness / FFTSubbandFlux / FFTSubbandPower

Goertzel // detect presence of a frequency

// You can also use filters to divide a signal into different ranges, and then detect energy for example in it

// >> more/advanced/experimental (ie: use at your own risk):

PV_CommonMag

MFCC

FrameCompare

Qitch

CQ_Diff

// These don't spit data, but re-synthesize for you:

PV_NoiseSynthF

PV_PartialSynthF

// ====== STATISTICAL UTILITIES ======

// It can be very useful, after analyzing a signal, to make a statistical analysis and get a better idea about its behavior

AverageOutput

TrigAvg

Max

RunningMax

RunningMin

RunningSum

Slope

Crest

StatUGens // PseudoUGens, built on RunningSum UGen; no help-file for the individual UGens

DynamicScaleUGen 

FluctuationUGen

KurtosisUGen

MaxDynScaleUGen

MinDynScaleUGen

SkewUGen

StdDevUGen

// ====== COMBINING FEATURES ======

// An example combining onset, amplitude and pitch analysis (by Josh P)

// This examples use the incoming audio from your soundcard - beware of feedback 

// Sing or whistle on the mic:

(

var freq, amp;

/* combining triggers - setting variable in the lang */

a = CtkSynthDef(\pitch, {arg inbus, attack, decay, id1, id2, id3, 

level = 6;

var in, amp, del, diff, freq, hasFreq, trig1, trig2, trig;

in = In.ar(inbus, 1);

amp = Amplitude.ar(in, attack, decay);

del = DelayN.ar(amp, 0.01, 0.01);

// delays the trigger itself

diff = DelayN.ar((amp / del.max(0.001)).ampdb * 

EnvGen.kr(Env([0, 0, 1], [0.1, 0.1])), 0.025, 0.025);

trig = Trig1.ar(diff > level, 1); // for amplitude

SendTrig.ar(trig, id1, RunningSum.kr(amp, 40)); // output the peak amplitude of the analysis

// send another trig with id2 for freq

#freq, hasFreq = Pitch.kr(in);

SendTrig.ar(trig, id2, freq);

// finally! send a last trigger with a slight delay, to ensure the 

// variables for the above two triggers have been set

SendTrig.ar(DelayN.ar(trig, 0.01, 0.01), id3, freq);

});

b = a.new.inbus_(s.options.numOutputBusChannels).level_(50)

.id1_(100).id2_(101).id3_(102).play;

z = CtkSynthDef(\test, {arg freq, amp;

Out.ar(1, SinOsc.ar(freq, 0, amp * XLine.kr(0.1, 0.0001, 4, doneAction: 2)))

});

o = OSCresponderNode(nil, '/tr', {arg time, resp, msg;

case 

{msg[2] == 100}

{amp = msg[3]; ("Amp is: "++amp).postln;}

{msg[2] == 101}

{freq = msg[3]; ("Freq is: "++freq).postln;}

{msg[2] == 102}

{

"Go!".postln;

// now - actually do something once the trigger happens

10.do({arg i;

z.new(0 + (i * 0.1.rand))

.freq_(freq * ((i + 1 * 2))).amp_(amp * 0.1).play;

})

}

}).add; 

)

b.level_(12);

b.free;

o.remove;